{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '3'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import json\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from unidecode import unidecode"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import bert\n",
    "from bert import run_classifier\n",
    "from bert import optimization\n",
    "from bert import tokenization\n",
    "from bert import modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import unicodedata\n",
    "import six\n",
    "from functools import partial\n",
    "\n",
    "SPIECE_UNDERLINE = '▁'\n",
    "\n",
    "def preprocess_text(inputs, lower=False, remove_space=True, keep_accents=False):\n",
    "  if remove_space:\n",
    "    outputs = ' '.join(inputs.strip().split())\n",
    "  else:\n",
    "    outputs = inputs\n",
    "  outputs = outputs.replace(\"``\", '\"').replace(\"''\", '\"')\n",
    "\n",
    "  if six.PY2 and isinstance(outputs, str):\n",
    "    outputs = outputs.decode('utf-8')\n",
    "\n",
    "  if not keep_accents:\n",
    "    outputs = unicodedata.normalize('NFKD', outputs)\n",
    "    outputs = ''.join([c for c in outputs if not unicodedata.combining(c)])\n",
    "  if lower:\n",
    "    outputs = outputs.lower()\n",
    "\n",
    "  return outputs\n",
    "\n",
    "\n",
    "def encode_pieces(sp_model, text, return_unicode=True, sample=False):\n",
    "  # return_unicode is used only for py2\n",
    "\n",
    "  # note(zhiliny): in some systems, sentencepiece only accepts str for py2\n",
    "  if six.PY2 and isinstance(text, unicode):\n",
    "    text = text.encode('utf-8')\n",
    "\n",
    "  if not sample:\n",
    "    pieces = sp_model.EncodeAsPieces(text)\n",
    "  else:\n",
    "    pieces = sp_model.SampleEncodeAsPieces(text, 64, 0.1)\n",
    "  new_pieces = []\n",
    "  for piece in pieces:\n",
    "    if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit():\n",
    "      cur_pieces = sp_model.EncodeAsPieces(\n",
    "          piece[:-1].replace(SPIECE_UNDERLINE, ''))\n",
    "      if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:\n",
    "        if len(cur_pieces[0]) == 1:\n",
    "          cur_pieces = cur_pieces[1:]\n",
    "        else:\n",
    "          cur_pieces[0] = cur_pieces[0][1:]\n",
    "      cur_pieces.append(piece[-1])\n",
    "      new_pieces.extend(cur_pieces)\n",
    "    else:\n",
    "      new_pieces.append(piece)\n",
    "\n",
    "  # note(zhiliny): convert back to unicode for py2\n",
    "  if six.PY2 and return_unicode:\n",
    "    ret_pieces = []\n",
    "    for piece in new_pieces:\n",
    "      if isinstance(piece, str):\n",
    "        piece = piece.decode('utf-8')\n",
    "      ret_pieces.append(piece)\n",
    "    new_pieces = ret_pieces\n",
    "\n",
    "  return new_pieces\n",
    "\n",
    "\n",
    "def encode_ids(sp_model, text, sample=False):\n",
    "  pieces = encode_pieces(sp_model, text, return_unicode=False, sample=sample)\n",
    "  ids = [sp_model.PieceToId(piece) for piece in pieces]\n",
    "  return ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malaya-model/bert-bahasa/bert-base-2020-03-19.tar.gz\n",
    "# !tar -zxf bert-base-2020-03-19.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert_config.json\t\t\tmodel.ckpt-2000002.meta\r\n",
      "model.ckpt-2000002.data-00000-of-00001\tsp10m.cased.v9.model\r\n",
      "model.ckpt-2000002.index\t\tsp10m.cased.v9.vocab\r\n"
     ]
    }
   ],
   "source": [
    "!ls bert-base-2020-03-19"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malaya/master/pretrained-model/preprocess/sp10m.cased.bert.model\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malaya/master/pretrained-model/preprocess/sp10m.cased.bert.vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "sp_model = spm.SentencePieceProcessor()\n",
    "sp_model.Load('sp10m.cased.bert.model')\n",
    "\n",
    "with open('sp10m.cased.bert.vocab') as fopen:\n",
    "    v = fopen.read().split('\\n')[:-1]\n",
    "v = [i.split('\\t') for i in v]\n",
    "v = {i[0]: i[1] for i in v}\n",
    "\n",
    "class Tokenizer:\n",
    "    def __init__(self, v):\n",
    "        self.vocab = v\n",
    "        pass\n",
    "    \n",
    "    def tokenize(self, string):\n",
    "        return encode_pieces(sp_model, string, return_unicode=False, sample=False)\n",
    "    \n",
    "    def convert_tokens_to_ids(self, tokens):\n",
    "        return [sp_model.PieceToId(piece) for piece in tokens]\n",
    "    \n",
    "    def convert_ids_to_tokens(self, ids):\n",
    "        return [sp_model.IdToPiece(i) for i in ids]\n",
    "    \n",
    "tokenizer = Tokenizer(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['train_X', 'train_Y', 'test_X', 'test_Y'])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "with open('ontonotes5-train-test.json') as fopen:\n",
    "    data = json.load(fopen)\n",
    "data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X = data['train_X']\n",
    "test_X = data['test_X']\n",
    "train_Y = data['train_Y']\n",
    "test_Y = data['test_Y']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "augmentation-org-ontonotes5.json\n",
      "35150 35150\n",
      "6300 6300\n",
      "augmentation-fac-ontonotes5.json\n",
      "17040 17040\n",
      "4260 4260\n",
      "augmentation-loc-ontonotes5.json\n",
      "13040 13040\n",
      "2460 2460\n",
      "augmentation-gpe-ontonotes5.json\n",
      "21060 21060\n",
      "0 0\n",
      "augmentation-work-of-art-ontonotes5.json\n",
      "4020 4020\n",
      "1020 1020\n",
      "augmentation-event-ontonotes5.json\n",
      "1665 1665\n",
      "0 0\n",
      "augmentation-person-ontonotes5.json\n",
      "37883 37883\n",
      "2530 2530\n",
      "augmentation-product-ontonotes5.json\n",
      "7040 7040\n",
      "1760 1760\n",
      "augmentation-law-ontonotes5.json\n",
      "12584 12584\n",
      "3161 3161\n",
      "augmentation-language-ontonotes5.json\n",
      "6860 6860\n",
      "0 0\n",
      "augmentation-norp-ontonotes5.json\n",
      "30660 30660\n",
      "5940 5940\n",
      "augmentation-address-ontonotes5.json\n",
      "57502 57502\n",
      "15106 15106\n"
     ]
    }
   ],
   "source": [
    "from glob import glob\n",
    "\n",
    "augmented = glob('augmentation-*-ontonotes5.json')\n",
    "\n",
    "for f in augmented:\n",
    "    print(f)\n",
    "    with open(f) as fopen:\n",
    "        data = json.load(fopen)\n",
    "        \n",
    "    print(len(data.get('train_X', [])), len(data.get('train_Y', [])))\n",
    "    print(len(data.get('test_X', [])), len(data.get('test_Y', [])))\n",
    "    \n",
    "    train_X.extend(data.get('train_X', []))\n",
    "    train_Y.extend(data.get('train_Y', []))\n",
    "    test_X.extend(data.get('test_X', []))\n",
    "    test_Y.extend(data.get('test_Y', []))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = [\n",
    "    {'Tag': 'OTHER', 'Description': 'other'},\n",
    "    {'Tag': 'ADDRESS', 'Description': 'Address of physical location.'},\n",
    "    {'Tag': 'PERSON', 'Description': 'People, including fictional.'},\n",
    "    {\n",
    "        'Tag': 'NORP',\n",
    "        'Description': 'Nationalities or religious or political groups.',\n",
    "    },\n",
    "    {\n",
    "        'Tag': 'FAC',\n",
    "        'Description': 'Buildings, airports, highways, bridges, etc.',\n",
    "    },\n",
    "    {\n",
    "        'Tag': 'ORG',\n",
    "        'Description': 'Companies, agencies, institutions, etc.',\n",
    "    },\n",
    "    {'Tag': 'GPE', 'Description': 'Countries, cities, states.'},\n",
    "    {\n",
    "        'Tag': 'LOC',\n",
    "        'Description': 'Non-GPE locations, mountain ranges, bodies of water.',\n",
    "    },\n",
    "    {\n",
    "        'Tag': 'PRODUCT',\n",
    "        'Description': 'Objects, vehicles, foods, etc. (Not services.)',\n",
    "    },\n",
    "    {\n",
    "        'Tag': 'EVENT',\n",
    "        'Description': 'Named hurricanes, battles, wars, sports events, etc.',\n",
    "    },\n",
    "    {'Tag': 'WORK_OF_ART', 'Description': 'Titles of books, songs, etc.'},\n",
    "    {'Tag': 'LAW', 'Description': 'Named documents made into laws.'},\n",
    "    {'Tag': 'LANGUAGE', 'Description': 'Any named language.'},\n",
    "    {\n",
    "        'Tag': 'DATE',\n",
    "        'Description': 'Absolute or relative dates or periods.',\n",
    "    },\n",
    "    {'Tag': 'TIME', 'Description': 'Times smaller than a day.'},\n",
    "    {'Tag': 'PERCENT', 'Description': 'Percentage, including \"%\".'},\n",
    "    {'Tag': 'MONEY', 'Description': 'Monetary values, including unit.'},\n",
    "    {\n",
    "        'Tag': 'QUANTITY',\n",
    "        'Description': 'Measurements, as of weight or distance.',\n",
    "    },\n",
    "    {'Tag': 'ORDINAL', 'Description': '\"first\", \"second\", etc.'},\n",
    "    {\n",
    "        'Tag': 'CARDINAL',\n",
    "        'Description': 'Numerals that do not fall under another type.',\n",
    "    },\n",
    "]\n",
    "d = [d['Tag'] for d in d]\n",
    "d = ['PAD', 'X'] + d\n",
    "tag2idx = {i: no for no, i in enumerate(d)}\n",
    "idx2tag = {no: i for no, i in enumerate(d)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def XY(strings):\n",
    "    left_train, right_train = strings[0], strings[1]\n",
    "    X, Y, MASK = [], [], []\n",
    "    for i in tqdm(range(len(left_train))):\n",
    "        left = [d for d in left_train[i]]\n",
    "        right = [d for d in right_train[i]]\n",
    "        bert_tokens = ['[CLS]']\n",
    "        y = ['PAD']\n",
    "        for no, orig_token in enumerate(left):\n",
    "            t = tokenizer.tokenize(orig_token)\n",
    "            bert_tokens.extend(t)\n",
    "            if len(t):\n",
    "                y.append(right[no])\n",
    "            y.extend(['X'] * (len(t) - 1))\n",
    "        bert_tokens.append('[SEP]')\n",
    "        y.append('PAD')\n",
    "        x = tokenizer.convert_tokens_to_ids(bert_tokens)\n",
    "        y = [tag2idx[i] for i in y]\n",
    "        input_mask = [1] * len(y)\n",
    "        if len(x) != len(y):\n",
    "            print(i)\n",
    "        X.append(x)\n",
    "        Y.append(y)\n",
    "        MASK.append(input_mask)\n",
    "    return [(X, Y, MASK,)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 40812/40812 [00:24<00:00, 1650.02it/s]\n",
      "100%|██████████| 1/1 [00:00<00:00, 1236.89it/s].14it/s]\n",
      "100%|██████████| 40812/40812 [00:24<00:00, 1637.13it/s]\n",
      " 40%|████      | 16509/40812 [00:10<00:16, 1515.38it/s]\n",
      "100%|██████████| 40812/40812 [00:24<00:00, 1644.78it/s]\n",
      "100%|██████████| 40812/40812 [00:24<00:00, 1655.85it/s]\n",
      "100%|██████████| 40812/40812 [00:25<00:00, 1606.47it/s]\n",
      "100%|██████████| 40812/40812 [00:24<00:00, 1673.43it/s]\n",
      "100%|██████████| 40812/40812 [00:24<00:00, 1674.92it/s]\n",
      "100%|██████████| 40812/40812 [00:25<00:00, 1595.99it/s]\n",
      "100%|██████████| 40812/40812 [00:25<00:00, 1612.04it/s]\n",
      "100%|██████████| 40812/40812 [00:26<00:00, 1538.24it/s]\n",
      "100%|██████████| 40812/40812 [00:25<00:00, 1575.62it/s]\n",
      "100%|██████████| 40812/40812 [00:26<00:00, 1560.13it/s]\n",
      "100%|██████████| 40812/40812 [00:26<00:00, 1561.15it/s]\n",
      "100%|██████████| 40812/40812 [00:29<00:00, 1405.82it/s]\n",
      "100%|██████████| 40812/40812 [00:27<00:00, 1475.96it/s]\n"
     ]
    }
   ],
   "source": [
    "import cleaning\n",
    "t = cleaning.multiprocessing(train_X, train_Y, XY)\n",
    "train_X, train_Y, train_masks = [], [], []\n",
    "for x, y, m in t:\n",
    "    train_X.extend(x)\n",
    "    train_Y.extend(y)\n",
    "    train_masks.extend(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5494/5494 [00:03<00:00, 1501.47it/s]\n",
      " 94%|█████████▍| 5186/5494 [00:03<00:00, 1314.24it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1611.70it/s]\n",
      " 64%|██████▍   | 3506/5494 [00:02<00:01, 1574.45it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1521.03it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1663.32it/s]\n",
      " 76%|███████▌  | 4157/5494 [00:02<00:00, 1524.00it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1542.82it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1528.34it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1537.33it/s]\n",
      " 96%|█████████▌| 5263/5494 [00:03<00:00, 1416.31it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1536.99it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1471.49it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1419.56it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1461.35it/s]\n",
      "100%|██████████| 5494/5494 [00:04<00:00, 1328.59it/s]\n",
      "100%|██████████| 5494/5494 [00:03<00:00, 1396.69it/s]\n"
     ]
    }
   ],
   "source": [
    "t = cleaning.multiprocessing(test_X, test_Y, XY)\n",
    "test_X, test_Y, test_masks = [], [], []\n",
    "for x, y, m in t:\n",
    "    test_X.extend(x)\n",
    "    test_Y.extend(y)\n",
    "    test_masks.extend(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "\n",
    "train_X, train_Y, train_masks = shuffle(train_X, train_Y, train_masks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "BERT_INIT_CHKPNT = 'bert-base-2020-03-19/model.ckpt-2000002'\n",
    "BERT_CONFIG = 'bert-base-2020-03-19/bert_config.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 5\n",
    "batch_size = 32\n",
    "warmup_proportion = 0.1\n",
    "num_train_steps = int(1000 / batch_size * epoch)\n",
    "num_warmup_steps = int(num_train_steps * warmup_proportion)\n",
    "bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_initializer(initializer_range=0.02):\n",
    "    return tf.truncated_normal_initializer(stddev=initializer_range)\n",
    "\n",
    "class Model:\n",
    "    def __init__(\n",
    "        self,\n",
    "        dimension_output,\n",
    "        learning_rate = 2e-5,\n",
    "        training = True\n",
    "    ):\n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.MASK = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.maxlen = tf.shape(self.X)[1]\n",
    "        self.lengths = tf.count_nonzero(self.X, 1)\n",
    "        \n",
    "        model = modeling.BertModel(\n",
    "            config=bert_config,\n",
    "            is_training=training,\n",
    "            input_ids=self.X,\n",
    "            input_mask=self.MASK,\n",
    "            use_one_hot_embeddings=False)\n",
    "        output_layer = model.get_sequence_output()\n",
    "        output_layer = tf.layers.dense(\n",
    "            output_layer,\n",
    "            bert_config.hidden_size,\n",
    "            activation=tf.tanh,\n",
    "            kernel_initializer=create_initializer())\n",
    "        logits = tf.layers.dense(output_layer, dimension_output,\n",
    "                                         kernel_initializer=create_initializer())\n",
    "        y_t = self.Y\n",
    "        log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(\n",
    "            logits, y_t, self.lengths\n",
    "        )\n",
    "        self.cost = tf.reduce_mean(-log_likelihood)\n",
    "        self.optimizer = tf.train.AdamOptimizer(\n",
    "            learning_rate = learning_rate\n",
    "        ).minimize(self.cost)\n",
    "        mask = tf.sequence_mask(self.lengths, maxlen = self.maxlen)\n",
    "        self.tags_seq, tags_score = tf.contrib.crf.crf_decode(\n",
    "            logits, transition_params, self.lengths\n",
    "        )\n",
    "        self.tags_seq = tf.identity(self.tags_seq, name = 'logits')\n",
    "\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(self.tags_seq, mask)\n",
    "        mask_label = tf.boolean_mask(y_t, mask)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/bert/modeling.py:233: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:187: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/crf/python/ops/crf.py:99: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/contrib/crf/python/ops/crf.py:213: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "INFO:tensorflow:Restoring parameters from bert-base-2020-03-19/model.ckpt-2000002\n"
     ]
    }
   ],
   "source": [
    "dimension_output = len(tag2idx)\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "var_lists = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope = 'bert')\n",
    "saver = tf.train.Saver(var_list = var_lists)\n",
    "saver.restore(sess, BERT_INIT_CHKPNT)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def merge_sentencepiece_tokens_tagging(x, y):\n",
    "    new_paired_tokens = []\n",
    "    n_tokens = len(x)\n",
    "    rejected = ['[CLS]', '[SEP]']\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    while i < n_tokens:\n",
    "\n",
    "        current_token, current_label = x[i], y[i]\n",
    "        if not current_token.startswith('▁') and current_token not in rejected:\n",
    "            previous_token, previous_label = new_paired_tokens.pop()\n",
    "            merged_token = previous_token\n",
    "            merged_label = [previous_label]\n",
    "            while (\n",
    "                not current_token.startswith('▁')\n",
    "                and current_token not in rejected\n",
    "            ):\n",
    "                merged_token = merged_token + current_token.replace('▁', '')\n",
    "                merged_label.append(current_label)\n",
    "                i = i + 1\n",
    "                current_token, current_label = x[i], y[i]\n",
    "            merged_label = merged_label[0]\n",
    "            new_paired_tokens.append((merged_token, merged_label))\n",
    "\n",
    "        else:\n",
    "            new_paired_tokens.append((current_token, current_label))\n",
    "            i = i + 1\n",
    "\n",
    "    words = [\n",
    "        i[0].replace('▁', '')\n",
    "        for i in new_paired_tokens\n",
    "        if i[0] not in rejected\n",
    "    ]\n",
    "    labels = [i[1] for i in new_paired_tokens if i[0] not in rejected]\n",
    "    return words, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'KUALA LUMPUR: Sempena sambutan Aidilfitri minggu depan, Perdana Menteri Tun Dr Mahathir Mohamad dan Menteri Pengangkutan Anthony Loke Siew Fook menitipkan pesanan khas kepada orang ramai yang mahu pulang ke kampung halaman masing-masing. Dalam video pendek terbitan Jabatan Keselamatan Jalan Raya (JKJR) itu, Dr Mahathir menasihati mereka supaya berhenti berehat dan tidur sebentar  sekiranya mengantuk ketika memandu.'\n",
    "\n",
    "import re\n",
    "\n",
    "def entities_textcleaning(string, lowering = False):\n",
    "    \"\"\"\n",
    "    use by entities recognition, pos recognition and dependency parsing\n",
    "    \"\"\"\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    original_string = string.split()\n",
    "    if lowering:\n",
    "        string = string.lower()\n",
    "    string = [\n",
    "        (original_string[no], word.title() if word.isupper() else word)\n",
    "        for no, word in enumerate(string.split())\n",
    "        if len(word)\n",
    "    ]\n",
    "    return [s[0] for s in string], [s[1] for s in string]\n",
    "\n",
    "def parse_X(left):\n",
    "    bert_tokens = ['[CLS]']\n",
    "    for no, orig_token in enumerate(left):\n",
    "        t = tokenizer.tokenize(orig_token)\n",
    "        bert_tokens.extend(t)\n",
    "    bert_tokens.append(\"[SEP]\")\n",
    "    input_mask = [1] * len(bert_tokens)\n",
    "    return tokenizer.convert_tokens_to_ids(bert_tokens), bert_tokens, input_mask\n",
    "\n",
    "sequence = entities_textcleaning(string)[1]\n",
    "parsed_sequence, bert_sequence, input_mask = parse_X(sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Kuala', 'PAD'),\n",
       " ('Lumpur:', 'TIME'),\n",
       " ('Sempena', 'CARDINAL'),\n",
       " ('sambutan', 'CARDINAL'),\n",
       " ('Aidilfitri', 'LOC'),\n",
       " ('minggu', 'GPE'),\n",
       " ('depan,', 'CARDINAL'),\n",
       " ('Perdana', 'PRODUCT'),\n",
       " ('Menteri', 'LOC'),\n",
       " ('Tun', 'OTHER'),\n",
       " ('Dr', 'OTHER'),\n",
       " ('Mahathir', 'PERCENT'),\n",
       " ('Mohamad', 'GPE'),\n",
       " ('dan', 'NORP'),\n",
       " ('Menteri', 'OTHER'),\n",
       " ('Pengangkutan', 'OTHER'),\n",
       " ('Anthony', 'TIME'),\n",
       " ('Loke', 'PERSON'),\n",
       " ('Siew', 'GPE'),\n",
       " ('Fook', 'ADDRESS'),\n",
       " ('menitipkan', 'LAW'),\n",
       " ('pesanan', 'LAW'),\n",
       " ('khas', 'LAW'),\n",
       " ('kepada', 'LAW'),\n",
       " ('orang', 'FAC'),\n",
       " ('ramai', 'MONEY'),\n",
       " ('yang', 'LAW'),\n",
       " ('mahu', 'LAW'),\n",
       " ('pulang', 'LAW'),\n",
       " ('ke', 'NORP'),\n",
       " ('kampung', 'OTHER'),\n",
       " ('halaman', 'OTHER'),\n",
       " ('masing-masing.', 'OTHER'),\n",
       " ('Dalam', 'PRODUCT'),\n",
       " ('video', 'LAW'),\n",
       " ('pendek', 'FAC'),\n",
       " ('terbitan', 'LOC'),\n",
       " ('Jabatan', 'OTHER'),\n",
       " ('Keselamatan', 'TIME'),\n",
       " ('Jalan', 'PERSON'),\n",
       " ('Raya', 'X'),\n",
       " ('(Jkjr)', 'PERSON'),\n",
       " ('itu,', 'PERSON'),\n",
       " ('Dr', 'NORP'),\n",
       " ('Mahathir', 'OTHER'),\n",
       " ('menasihati', 'OTHER'),\n",
       " ('mereka', 'OTHER'),\n",
       " ('supaya', 'OTHER'),\n",
       " ('berhenti', 'OTHER'),\n",
       " ('berehat', 'ORDINAL'),\n",
       " ('dan', 'QUANTITY'),\n",
       " ('tidur', 'LOC'),\n",
       " ('sebentar', 'ADDRESS'),\n",
       " ('sekiranya', 'QUANTITY'),\n",
       " ('mengantuk', 'LAW'),\n",
       " ('ketika', 'X'),\n",
       " ('memandu.', 'ORDINAL')]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predicted = sess.run(model.tags_seq,\n",
    "                feed_dict = {\n",
    "                    model.X: [parsed_sequence],\n",
    "                    model.MASK: [input_mask]\n",
    "                },\n",
    "        )[0]\n",
    "merged = merge_sentencepiece_tokens_tagging(bert_sequence, [idx2tag[d] for d in predicted])\n",
    "list(zip(merged[0], merged[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 20407/20407 [2:06:31<00:00,  2.69it/s, accuracy=1, cost=0.705]      \n",
      "test minibatch loop: 100%|██████████| 2748/2748 [07:59<00:00,  5.73it/s, accuracy=0.999, cost=0.736] \n",
      "train minibatch loop:   0%|          | 0/20407 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 8070.775153160095\n",
      "epoch: 0, training loss: 2.863052, training acc: 0.986971, valid loss: 4.914965, valid acc: 0.983747\n",
      "\n",
      "[('Kuala', 'TIME'), ('Lumpur:', 'TIME'), ('Sempena', 'OTHER'), ('sambutan', 'OTHER'), ('Aidilfitri', 'DATE'), ('minggu', 'DATE'), ('depan,', 'DATE'), ('Perdana', 'OTHER'), ('Menteri', 'OTHER'), ('Tun', 'ORG'), ('Dr', 'PERSON'), ('Mahathir', 'PERSON'), ('Mohamad', 'PERSON'), ('dan', 'OTHER'), ('Menteri', 'OTHER'), ('Pengangkutan', 'OTHER'), ('Anthony', 'PERSON'), ('Loke', 'PERSON'), ('Siew', 'PERSON'), ('Fook', 'PERSON'), ('menitipkan', 'OTHER'), ('pesanan', 'OTHER'), ('khas', 'OTHER'), ('kepada', 'OTHER'), ('orang', 'OTHER'), ('ramai', 'OTHER'), ('yang', 'OTHER'), ('mahu', 'OTHER'), ('pulang', 'OTHER'), ('ke', 'OTHER'), ('kampung', 'OTHER'), ('halaman', 'OTHER'), ('masing-masing.', 'OTHER'), ('Dalam', 'OTHER'), ('video', 'OTHER'), ('pendek', 'OTHER'), ('terbitan', 'OTHER'), ('Jabatan', 'ORG'), ('Keselamatan', 'ORG'), ('Jalan', 'ORG'), ('Raya', 'ORG'), ('(Jkjr)', 'ORG'), ('itu,', 'OTHER'), ('Dr', 'PERSON'), ('Mahathir', 'PERSON'), ('menasihati', 'OTHER'), ('mereka', 'OTHER'), ('supaya', 'OTHER'), ('berhenti', 'OTHER'), ('berehat', 'OTHER'), ('dan', 'OTHER'), ('tidur', 'OTHER'), ('sebentar', 'OTHER'), ('sekiranya', 'OTHER'), ('mengantuk', 'OTHER'), ('ketika', 'OTHER'), ('memandu.', 'OTHER')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 20407/20407 [2:06:29<00:00,  2.69it/s, accuracy=1, cost=0.00201]    \n",
      "test minibatch loop: 100%|██████████| 2748/2748 [07:57<00:00,  5.76it/s, accuracy=1, cost=0.0769]    \n",
      "train minibatch loop:   0%|          | 0/20407 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 8066.735553979874\n",
      "epoch: 1, training loss: 0.540903, training acc: 0.997296, valid loss: 5.464173, valid acc: 0.984979\n",
      "\n",
      "[('Kuala', 'PERSON'), ('Lumpur:', 'TIME'), ('Sempena', 'OTHER'), ('sambutan', 'DATE'), ('Aidilfitri', 'DATE'), ('minggu', 'DATE'), ('depan,', 'DATE'), ('Perdana', 'PERSON'), ('Menteri', 'PERSON'), ('Tun', 'PERSON'), ('Dr', 'PERSON'), ('Mahathir', 'PERSON'), ('Mohamad', 'PERSON'), ('dan', 'OTHER'), ('Menteri', 'OTHER'), ('Pengangkutan', 'OTHER'), ('Anthony', 'PERSON'), ('Loke', 'PERSON'), ('Siew', 'PERSON'), ('Fook', 'PERSON'), ('menitipkan', 'OTHER'), ('pesanan', 'OTHER'), ('khas', 'OTHER'), ('kepada', 'OTHER'), ('orang', 'OTHER'), ('ramai', 'OTHER'), ('yang', 'OTHER'), ('mahu', 'OTHER'), ('pulang', 'OTHER'), ('ke', 'OTHER'), ('kampung', 'OTHER'), ('halaman', 'OTHER'), ('masing-masing.', 'OTHER'), ('Dalam', 'OTHER'), ('video', 'OTHER'), ('pendek', 'OTHER'), ('terbitan', 'OTHER'), ('Jabatan', 'ORG'), ('Keselamatan', 'ORG'), ('Jalan', 'ORG'), ('Raya', 'ORG'), ('(Jkjr)', 'ORG'), ('itu,', 'ORG'), ('Dr', 'PERSON'), ('Mahathir', 'PERSON'), ('menasihati', 'OTHER'), ('mereka', 'OTHER'), ('supaya', 'OTHER'), ('berhenti', 'OTHER'), ('berehat', 'OTHER'), ('dan', 'OTHER'), ('tidur', 'OTHER'), ('sebentar', 'OTHER'), ('sekiranya', 'OTHER'), ('mengantuk', 'OTHER'), ('ketika', 'OTHER'), ('memandu.', 'OTHER')]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop:   4%|▎         | 718/20407 [04:24<2:01:03,  2.71it/s, accuracy=0.996, cost=0.735]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-34-08fdd4304cbb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     21\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mX\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_x\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m                 \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mY\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_y\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 23\u001b[0;31m                 \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMASK\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mbatch_masks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     24\u001b[0m             },\n\u001b[1;32m     25\u001b[0m         )\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    954\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    955\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 956\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    957\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    958\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1178\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1179\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1180\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1181\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1182\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1357\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1359\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1360\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1361\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1363\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1364\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1365\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1366\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1367\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._call_tf_sessionrun(options, feed_dict, fetch_list,\n\u001b[0;32m-> 1350\u001b[0;31m                                       target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1441\u001b[0m     return tf_session.TF_SessionRun_wrapper(self._session, options, feed_dict,\n\u001b[1;32m   1442\u001b[0m                                             \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1443\u001b[0;31m                                             run_metadata)\n\u001b[0m\u001b[1;32m   1444\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1445\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "for e in range(epoch):\n",
    "    lasttime = time.time()\n",
    "    train_acc, train_loss, test_acc, test_loss = [], [], [], []\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'train minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i : index]\n",
    "        batch_y = train_Y[i : index]\n",
    "        batch_masks = train_masks[i : index]\n",
    "        batch_x = pad_sequences(batch_x, padding='post')\n",
    "        batch_y = pad_sequences(batch_y, padding='post')\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post')\n",
    "        \n",
    "        acc, cost, _ = sess.run(\n",
    "            [model.accuracy, model.cost, model.optimizer],\n",
    "            feed_dict = {\n",
    "                model.X: batch_x,\n",
    "                model.Y: batch_y,\n",
    "                model.MASK: batch_masks,\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        train_loss.append(cost)\n",
    "        train_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'test minibatch loop'\n",
    "    )\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = test_X[i : index]\n",
    "        batch_y = test_Y[i : index]\n",
    "        batch_masks = test_masks[i : index]\n",
    "        batch_x = pad_sequences(batch_x, padding='post')\n",
    "        batch_y = pad_sequences(batch_y, padding='post')\n",
    "        batch_masks = pad_sequences(batch_masks, padding='post')\n",
    "        \n",
    "        acc, cost = sess.run(\n",
    "            [model.accuracy, model.cost],\n",
    "            feed_dict = {\n",
    "                model.X: batch_x,\n",
    "                model.Y: batch_y,\n",
    "                model.MASK: batch_masks,\n",
    "            },\n",
    "        )\n",
    "        assert not np.isnan(cost)\n",
    "        test_loss.append(cost)\n",
    "        test_acc.append(acc)\n",
    "        pbar.set_postfix(cost = cost, accuracy = acc)\n",
    "    \n",
    "    train_loss = np.mean(train_loss)\n",
    "    train_acc = np.mean(train_acc)\n",
    "    test_loss = np.mean(test_loss)\n",
    "    test_acc = np.mean(test_acc)\n",
    "\n",
    "    print('time taken:', time.time() - lasttime)\n",
    "    print(\n",
    "        'epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'\n",
    "        % (e, train_loss, train_acc, test_loss, test_acc)\n",
    "    )\n",
    "    predicted = sess.run(model.tags_seq,\n",
    "                feed_dict = {\n",
    "                    model.X: [parsed_sequence],\n",
    "                    model.MASK: [input_mask]\n",
    "                },\n",
    "        )[0]\n",
    "    merged = merge_sentencepiece_tokens_tagging(bert_sequence, [idx2tag[d] for d in predicted])\n",
    "    print(list(zip(merged[0], merged[1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bert-base-entities/model.ckpt'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'bert-base-entities/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-entities/model.ckpt\n"
     ]
    }
   ],
   "source": [
    "dimension_output = len(tag2idx)\n",
    "learning_rate = 2e-5\n",
    "\n",
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(\n",
    "    dimension_output,\n",
    "    learning_rate,\n",
    "    training = False\n",
    ")\n",
    "\n",
    "sess.run(tf.global_variables_initializer())\n",
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.restore(sess, 'bert-base-entities/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pred2label(pred):\n",
    "    out = []\n",
    "    for pred_i in pred:\n",
    "        out_i = []\n",
    "        for p in pred_i:\n",
    "            out_i.append(idx2tag[p])\n",
    "        out.append(out_i)\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "validation minibatch loop: 100%|██████████| 2748/2748 [07:45<00:00,  5.91it/s]\n"
     ]
    }
   ],
   "source": [
    "real_Y, predict_Y = [], []\n",
    "\n",
    "pbar = tqdm(\n",
    "    range(0, len(test_X), batch_size), desc = 'validation minibatch loop'\n",
    ")\n",
    "for i in pbar:\n",
    "    index = min(i + batch_size, len(test_X))\n",
    "    batch_x = test_X[i : index]\n",
    "    batch_y = test_Y[i : index]\n",
    "    batch_masks = test_masks[i : index]\n",
    "    batch_x = pad_sequences(batch_x, padding='post')\n",
    "    batch_y = pad_sequences(batch_y, padding='post')\n",
    "    batch_masks = pad_sequences(batch_masks, padding='post')\n",
    "    predicted = pred2label(sess.run(model.tags_seq,\n",
    "            feed_dict = {\n",
    "                model.X: batch_x,\n",
    "                model.MASK: batch_masks,\n",
    "            },\n",
    "    ))\n",
    "    real = pred2label(batch_y)\n",
    "    predict_Y.extend(predicted)\n",
    "    real_Y.extend(real)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_real_Y = []\n",
    "for r in real_Y:\n",
    "    temp_real_Y.extend(r)\n",
    "    \n",
    "temp_predict_Y = []\n",
    "for r in predict_Y:\n",
    "    temp_predict_Y.extend(r)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "     ADDRESS    0.99858   0.99974   0.99916     93446\n",
      "    CARDINAL    0.93840   0.90631   0.92207     48255\n",
      "        DATE    0.95490   0.93656   0.94564    126548\n",
      "       EVENT    0.92876   0.93591   0.93232      5711\n",
      "         FAC    0.93271   0.92658   0.92964     27392\n",
      "         GPE    0.93437   0.94852   0.94139    101357\n",
      "    LANGUAGE    0.93478   0.96389   0.94911       803\n",
      "         LAW    0.94824   0.95744   0.95281     24834\n",
      "         LOC    0.94148   0.93213   0.93678     34538\n",
      "       MONEY    0.87803   0.87563   0.87683     30032\n",
      "        NORP    0.95516   0.90446   0.92912     57014\n",
      "     ORDINAL    0.91510   0.91083   0.91296      6213\n",
      "         ORG    0.92453   0.95354   0.93881    219533\n",
      "       OTHER    0.99135   0.99308   0.99221   3553350\n",
      "         PAD    0.99956   1.00000   0.99978   1292421\n",
      "     PERCENT    0.96287   0.96814   0.96550     21722\n",
      "      PERSON    0.97376   0.93891   0.95602    101981\n",
      "     PRODUCT    0.87537   0.81769   0.84555     11124\n",
      "    QUANTITY    0.94385   0.92483   0.93424     11614\n",
      "        TIME    0.91912   0.90170   0.91033      9502\n",
      " WORK_OF_ART    0.93126   0.81978   0.87197     13800\n",
      "           X    0.99906   0.99792   0.99849   1350434\n",
      "\n",
      "    accuracy                        0.98821   7141624\n",
      "   macro avg    0.94460   0.93244   0.93822   7141624\n",
      "weighted avg    0.98821   0.98821   0.98818   7141624\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(temp_real_Y, temp_predict_Y, digits = 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Placeholder',\n",
       " 'Placeholder_1',\n",
       " 'Placeholder_2',\n",
       " 'bert/embeddings/word_embeddings',\n",
       " 'bert/embeddings/token_type_embeddings',\n",
       " 'bert/embeddings/position_embeddings',\n",
       " 'bert/embeddings/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/query/bias',\n",
       " 'bert/encoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/key/bias',\n",
       " 'bert/encoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/value/bias',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax',\n",
       " 'bert/encoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_0/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/output/dense/bias',\n",
       " 'bert/encoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/query/bias',\n",
       " 'bert/encoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/key/bias',\n",
       " 'bert/encoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/value/bias',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax',\n",
       " 'bert/encoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_1/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/output/dense/bias',\n",
       " 'bert/encoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/query/bias',\n",
       " 'bert/encoder/layer_2/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/key/bias',\n",
       " 'bert/encoder/layer_2/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_2/attention/self/value/bias',\n",
       " 'bert/encoder/layer_2/attention/self/Softmax',\n",
       " 'bert/encoder/layer_2/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_2/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_2/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_2/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_2/output/dense/kernel',\n",
       " 'bert/encoder/layer_2/output/dense/bias',\n",
       " 'bert/encoder/layer_2/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/query/bias',\n",
       " 'bert/encoder/layer_3/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/key/bias',\n",
       " 'bert/encoder/layer_3/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_3/attention/self/value/bias',\n",
       " 'bert/encoder/layer_3/attention/self/Softmax',\n",
       " 'bert/encoder/layer_3/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_3/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_3/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_3/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_3/output/dense/kernel',\n",
       " 'bert/encoder/layer_3/output/dense/bias',\n",
       " 'bert/encoder/layer_3/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_4/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/query/bias',\n",
       " 'bert/encoder/layer_4/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/key/bias',\n",
       " 'bert/encoder/layer_4/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_4/attention/self/value/bias',\n",
       " 'bert/encoder/layer_4/attention/self/Softmax',\n",
       " 'bert/encoder/layer_4/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_4/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_4/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_4/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_4/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_4/output/dense/kernel',\n",
       " 'bert/encoder/layer_4/output/dense/bias',\n",
       " 'bert/encoder/layer_4/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_5/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/query/bias',\n",
       " 'bert/encoder/layer_5/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/key/bias',\n",
       " 'bert/encoder/layer_5/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_5/attention/self/value/bias',\n",
       " 'bert/encoder/layer_5/attention/self/Softmax',\n",
       " 'bert/encoder/layer_5/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_5/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_5/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_5/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_5/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_5/output/dense/kernel',\n",
       " 'bert/encoder/layer_5/output/dense/bias',\n",
       " 'bert/encoder/layer_5/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_6/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_6/attention/self/query/bias',\n",
       " 'bert/encoder/layer_6/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_6/attention/self/key/bias',\n",
       " 'bert/encoder/layer_6/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_6/attention/self/value/bias',\n",
       " 'bert/encoder/layer_6/attention/self/Softmax',\n",
       " 'bert/encoder/layer_6/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_6/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_6/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_6/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_6/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_6/output/dense/kernel',\n",
       " 'bert/encoder/layer_6/output/dense/bias',\n",
       " 'bert/encoder/layer_6/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_7/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_7/attention/self/query/bias',\n",
       " 'bert/encoder/layer_7/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_7/attention/self/key/bias',\n",
       " 'bert/encoder/layer_7/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_7/attention/self/value/bias',\n",
       " 'bert/encoder/layer_7/attention/self/Softmax',\n",
       " 'bert/encoder/layer_7/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_7/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_7/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_7/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_7/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_7/output/dense/kernel',\n",
       " 'bert/encoder/layer_7/output/dense/bias',\n",
       " 'bert/encoder/layer_7/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_8/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_8/attention/self/query/bias',\n",
       " 'bert/encoder/layer_8/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_8/attention/self/key/bias',\n",
       " 'bert/encoder/layer_8/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_8/attention/self/value/bias',\n",
       " 'bert/encoder/layer_8/attention/self/Softmax',\n",
       " 'bert/encoder/layer_8/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_8/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_8/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_8/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_8/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_8/output/dense/kernel',\n",
       " 'bert/encoder/layer_8/output/dense/bias',\n",
       " 'bert/encoder/layer_8/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_9/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_9/attention/self/query/bias',\n",
       " 'bert/encoder/layer_9/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_9/attention/self/key/bias',\n",
       " 'bert/encoder/layer_9/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_9/attention/self/value/bias',\n",
       " 'bert/encoder/layer_9/attention/self/Softmax',\n",
       " 'bert/encoder/layer_9/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_9/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_9/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_9/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_9/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_9/output/dense/kernel',\n",
       " 'bert/encoder/layer_9/output/dense/bias',\n",
       " 'bert/encoder/layer_9/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_10/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_10/attention/self/query/bias',\n",
       " 'bert/encoder/layer_10/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_10/attention/self/key/bias',\n",
       " 'bert/encoder/layer_10/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_10/attention/self/value/bias',\n",
       " 'bert/encoder/layer_10/attention/self/Softmax',\n",
       " 'bert/encoder/layer_10/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_10/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_10/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_10/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_10/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_10/output/dense/kernel',\n",
       " 'bert/encoder/layer_10/output/dense/bias',\n",
       " 'bert/encoder/layer_10/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_11/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_11/attention/self/query/bias',\n",
       " 'bert/encoder/layer_11/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_11/attention/self/key/bias',\n",
       " 'bert/encoder/layer_11/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_11/attention/self/value/bias',\n",
       " 'bert/encoder/layer_11/attention/self/Softmax',\n",
       " 'bert/encoder/layer_11/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_11/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_11/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_11/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_11/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_11/output/dense/kernel',\n",
       " 'bert/encoder/layer_11/output/dense/bias',\n",
       " 'bert/encoder/layer_11/output/LayerNorm/gamma',\n",
       " 'bert/pooler/dense/kernel',\n",
       " 'bert/pooler/dense/bias',\n",
       " 'dense/kernel',\n",
       " 'dense/bias',\n",
       " 'dense_1/kernel',\n",
       " 'dense_1/bias',\n",
       " 'transitions',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_11/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_10/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_9/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_8/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_7/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_6/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_5/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_4/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_3/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_2/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_1/attention/self/Softmax_grad/mul_1',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum/reduction_indices',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/Sum',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/sub',\n",
       " 'gradients/bert/encoder/layer_0/attention/self/Softmax_grad/mul_1',\n",
       " 'logits']"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'Adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bert-base-entities/model.ckpt\n",
      "WARNING:tensorflow:From <ipython-input-43-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n",
      "INFO:tensorflow:Froze 204 variables.\n",
      "INFO:tensorflow:Converted 204 variables to const ops.\n",
      "13221 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('bert-base-entities', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/client/session.py:1750: UserWarning: An interactive session is already active. This can cause out-of-memory errors in some cases. You must explicitly call `InteractiveSession.close()` to release resources held by the other session(s).\n",
      "  warnings.warn('An interactive session is already active. This can '\n"
     ]
    }
   ],
   "source": [
    "def load_graph(frozen_graph_filename):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph\n",
    "\n",
    "g = load_graph('bert-base-entities/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "mask = g.get_tensor_by_name('import/Placeholder_1:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "string = 'Kyrgios, 25, membuat pesanan itu kerana menyedaari pelbagai kesukaran menimpa rakyat Australia ekoran perintah kawalan pergerakan yang diumumkan Mac lalu bagi memerangi wabak COVID-19 di negara berkenaan. Pemain tenis ranking ke-40 dunia yang dilahirkan di Canberra itu meminta pengikut dan penyokongnya agar jangan tidur dalam keadaan perut kosong dalam hantaran Instagram yang meraih lebih 92,000 tanda suka.'\n",
    "\n",
    "ori, sequence = entities_textcleaning(string)\n",
    "parsed_sequence, bert_sequence, input_mask = parse_X(sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('Kyrgios,', 'PERSON'), ('25,', 'NORP'), ('membuat', 'OTHER'), ('pesanan', 'OTHER'), ('itu', 'OTHER'), ('kerana', 'OTHER'), ('menyedaari', 'OTHER'), ('pelbagai', 'OTHER'), ('kesukaran', 'OTHER'), ('menimpa', 'OTHER'), ('rakyat', 'OTHER'), ('Australia', 'NORP'), ('ekoran', 'OTHER'), ('perintah', 'OTHER'), ('kawalan', 'OTHER'), ('pergerakan', 'OTHER'), ('yang', 'OTHER'), ('diumumkan', 'OTHER'), ('Mac', 'DATE'), ('lalu', 'DATE'), ('bagi', 'OTHER'), ('memerangi', 'OTHER'), ('wabak', 'OTHER'), ('Covid-19', 'OTHER'), ('di', 'OTHER'), ('negara', 'OTHER'), ('berkenaan.', 'OTHER'), ('Pemain', 'OTHER'), ('tenis', 'OTHER'), ('ranking', 'OTHER'), ('ke-40', 'ORDINAL'), ('dunia', 'OTHER'), ('yang', 'OTHER'), ('dilahirkan', 'OTHER'), ('di', 'OTHER'), ('Canberra', 'GPE'), ('itu', 'OTHER'), ('meminta', 'OTHER'), ('pengikut', 'OTHER'), ('dan', 'OTHER'), ('penyokongnya', 'OTHER'), ('agar', 'OTHER'), ('jangan', 'OTHER'), ('tidur', 'OTHER'), ('dalam', 'OTHER'), ('keadaan', 'OTHER'), ('perut', 'OTHER'), ('kosong', 'OTHER'), ('dalam', 'OTHER'), ('hantaran', 'OTHER'), ('Instagram', 'ORG'), ('yang', 'OTHER'), ('meraih', 'OTHER'), ('lebih', 'OTHER'), ('92,000', 'MONEY'), ('tanda', 'MONEY'), ('suka.', 'OTHER')]\n"
     ]
    }
   ],
   "source": [
    "predicted = test_sess.run(logits,\n",
    "            feed_dict = {\n",
    "                x: [parsed_sequence],\n",
    "                mask: [input_mask]\n",
    "            },\n",
    "    )[0]\n",
    "merged = merge_sentencepiece_tokens_tagging(bert_sequence, [idx2tag[d] for d in predicted])\n",
    "print(list(zip(merged[0], merged[1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "tf.set_random_seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-50-526b53248e16>:12: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n",
      "bert-base-entities/frozen_model.pb ['Placeholder', 'Placeholder_1']\n"
     ]
    }
   ],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']\n",
    "\n",
    "pb = 'bert-base-entities/frozen_model.pb'\n",
    "\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "\n",
    "inputs = ['Placeholder', 'Placeholder_1']\n",
    "outputs = ['dense/BiasAdd']\n",
    "\n",
    "print(pb, inputs)\n",
    "\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits'] + outputs, transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
